{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-26T05:23:52.060296Z",
     "iopub.status.busy": "2021-06-26T05:23:52.059890Z",
     "iopub.status.idle": "2021-06-26T05:23:52.298202Z",
     "shell.execute_reply": "2021-06-26T05:23:52.297175Z",
     "shell.execute_reply.started": "2021-06-26T05:23:52.060276Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import codecs\n",
    "import os, re, json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://storage.googleapis.com/cluebenchmark/tasks/cmrc2018_public.zip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-26T05:26:46.277729Z",
     "iopub.status.busy": "2021-06-26T05:26:46.277209Z",
     "iopub.status.idle": "2021-06-26T05:26:46.449522Z",
     "shell.execute_reply": "2021-06-26T05:26:46.448990Z",
     "shell.execute_reply.started": "2021-06-26T05:26:46.277684Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "train = json.load(open('./cmrc2018_public/train.json'))\n",
    "dev = json.load(open('./cmrc2018_public/dev.json'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-26T05:27:26.948165Z",
     "iopub.status.busy": "2021-06-26T05:27:26.947609Z",
     "iopub.status.idle": "2021-06-26T05:27:26.955585Z",
     "shell.execute_reply": "2021-06-26T05:27:26.954673Z",
     "shell.execute_reply.started": "2021-06-26T05:27:26.948121Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'id': 'TRAIN_186',\n",
       " 'context': '范廷颂枢机（，），圣名保禄·若瑟（），是越南罗马天主教枢机。1963年被任为主教；1990年被擢升为天主教河内总教区宗座署理；1994年被擢升为总主教，同年年底被擢升为枢机；2009年2月离世。范廷颂于1919年6月15日在越南宁平省天主教发艳教区出生；童年时接受良好教育后，被一位越南神父带到河内继续其学业。范廷颂于1940年在河内大修道院完成神学学业。范廷颂于1949年6月6日在河内的主教座堂晋铎；及后被派到圣女小德兰孤儿院服务。1950年代，范廷颂在河内堂区创建移民接待中心以收容到河内避战的难民。1954年，法越战争结束，越南民主共和国建都河内，当时很多天主教神职人员逃至越南的南方，但范廷颂仍然留在河内。翌年管理圣若望小修院；惟在1960年因捍卫修院的自由、自治及拒绝政府在修院设政治课的要求而被捕。1963年4月5日，教宗任命范廷颂为天主教北宁教区主教，同年8月15日就任；其牧铭为「我信天主的爱」。由于范廷颂被越南政府软禁差不多30年，因此他无法到所属堂区进行牧灵工作而专注研读等工作。范廷颂除了面对战争、贫困、被当局迫害天主教会等问题外，也秘密恢复修院、创建女修会团体等。1990年，教宗若望保禄二世在同年6月18日擢升范廷颂为天主教河内总教区宗座署理以填补该教区总主教的空缺。1994年3月23日，范廷颂被教宗若望保禄二世擢升为天主教河内总教区总主教并兼天主教谅山教区宗座署理；同年11月26日，若望保禄二世擢升范廷颂为枢机。范廷颂在1995年至2001年期间出任天主教越南主教团主席。2003年4月26日，教宗若望保禄二世任命天主教谅山教区兼天主教高平教区吴光杰主教为天主教河内总教区署理主教；及至2005年2月19日，范廷颂因获批辞去总主教职务而荣休；吴光杰同日真除天主教河内总教区总主教职务。范廷颂于2009年2月22日清晨在河内离世，享年89岁；其葬礼于同月26日上午在天主教河内总教区总主教座堂举行。',\n",
       " 'qas': [{'question': '范廷颂是什么时候被任为主教的？',\n",
       "   'id': 'TRAIN_186_QUERY_0',\n",
       "   'answers': [{'text': '1963年', 'answer_start': 30}]},\n",
       "  {'question': '1990年，范廷颂担任什么职务？',\n",
       "   'id': 'TRAIN_186_QUERY_1',\n",
       "   'answers': [{'text': '1990年被擢升为天主教河内总教区宗座署理', 'answer_start': 41}]},\n",
       "  {'question': '范廷颂是于何时何地出生的？',\n",
       "   'id': 'TRAIN_186_QUERY_2',\n",
       "   'answers': [{'text': '范廷颂于1919年6月15日在越南宁平省天主教发艳教区出生', 'answer_start': 97}]},\n",
       "  {'question': '1994年3月，范廷颂担任什么职务？',\n",
       "   'id': 'TRAIN_186_QUERY_3',\n",
       "   'answers': [{'text': '1994年3月23日，范廷颂被教宗若望保禄二世擢升为天主教河内总教区总主教并兼天主教谅山教区宗座署理',\n",
       "     'answer_start': 548}]},\n",
       "  {'question': '范廷颂是何时去世的？',\n",
       "   'id': 'TRAIN_186_QUERY_4',\n",
       "   'answers': [{'text': '范廷颂于2009年2月22日清晨在河内离世', 'answer_start': 759}]}]}"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train['data'][0]['paragraphs'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-26T05:30:36.956871Z",
     "iopub.status.busy": "2021-06-26T05:30:36.956304Z",
     "iopub.status.idle": "2021-06-26T05:30:50.398503Z",
     "shell.execute_reply": "2021-06-26T05:30:50.397997Z",
     "shell.execute_reply.started": "2021-06-26T05:30:36.956825Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at hfl/chinese-bert-wwm-ext were not used when initializing BertForQuestionAnswering: ['cls.predictions.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.decoder.weight', 'cls.seq_relationship.weight', 'cls.seq_relationship.bias']\n",
      "- This IS expected if you are initializing BertForQuestionAnswering from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertForQuestionAnswering from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of BertForQuestionAnswering were not initialized from the model checkpoint at hfl/chinese-bert-wwm-ext and are newly initialized: ['qa_outputs.weight', 'qa_outputs.bias']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "from transformers import BertTokenizerFast, BertForQuestionAnswering\n",
    "\n",
    "tokenizer = BertTokenizerFast.from_pretrained('hfl/chinese-bert-wwm-ext')\n",
    "model = BertForQuestionAnswering.from_pretrained('hfl/chinese-bert-wwm-ext')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-26T05:36:00.110264Z",
     "iopub.status.busy": "2021-06-26T05:36:00.109708Z",
     "iopub.status.idle": "2021-06-26T05:36:00.128612Z",
     "shell.execute_reply": "2021-06-26T05:36:00.127991Z",
     "shell.execute_reply.started": "2021-06-26T05:36:00.110217Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "paragraphs = []\n",
    "questions = []\n",
    "start_positions = []\n",
    "end_positions = []\n",
    "for paragraph in train['data']:\n",
    "    for qa in paragraph['paragraphs'][0]['qas']:\n",
    "        paragraphs.append(paragraph['paragraphs'][0]['context'])\n",
    "        questions.append(qa['question'])\n",
    "        start_positions.append(qa['answers'][0]['answer_start'])\n",
    "        end_positions.append(qa['answers'][0]['answer_start'] + len(qa['answers'][0]['text']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-26T05:36:00.482457Z",
     "iopub.status.busy": "2021-06-26T05:36:00.481935Z",
     "iopub.status.idle": "2021-06-26T05:36:02.617640Z",
     "shell.execute_reply": "2021-06-26T05:36:02.616480Z",
     "shell.execute_reply.started": "2021-06-26T05:36:00.482412Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "train_encodings = tokenizer(paragraphs, questions, \n",
    "                            return_tensors='pt', truncation=True, padding=True,\n",
    "                           max_length=512)\n",
    "\n",
    "train_encodings['start_positions'] = [train_encodings.char_to_token(idx, x) if train_encodings.char_to_token(idx, x) != None else -1\n",
    "                                      for idx, x in enumerate(start_positions)]\n",
    "train_encodings['end_positions'] = [train_encodings.char_to_token(idx, x-1) if train_encodings.char_to_token(idx, x-1) != None else -1\n",
    "                                    for idx, x in enumerate(end_positions)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-26T05:36:06.217371Z",
     "iopub.status.busy": "2021-06-26T05:36:06.216830Z",
     "iopub.status.idle": "2021-06-26T05:36:06.228958Z",
     "shell.execute_reply": "2021-06-26T05:36:06.228267Z",
     "shell.execute_reply.started": "2021-06-26T05:36:06.217325Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "paragraphs = []\n",
    "questions = []\n",
    "start_positions = []\n",
    "end_positions = []\n",
    "for paragraph in dev['data']:\n",
    "    for qa in paragraph['paragraphs'][0]['qas']:\n",
    "        paragraphs.append(paragraph['paragraphs'][0]['context'])\n",
    "        questions.append(qa['question'])\n",
    "        start_positions.append(qa['answers'][0]['answer_start'])\n",
    "        end_positions.append(qa['answers'][0]['answer_start'] + len(qa['answers'][0]['text']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-26T05:36:06.553486Z",
     "iopub.status.busy": "2021-06-26T05:36:06.552976Z",
     "iopub.status.idle": "2021-06-26T05:36:07.316741Z",
     "shell.execute_reply": "2021-06-26T05:36:07.316196Z",
     "shell.execute_reply.started": "2021-06-26T05:36:06.553441Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "val_encodings = tokenizer(paragraphs, questions, \n",
    "                            return_tensors='pt', truncation=True, padding=True,\n",
    "                           max_length=512)\n",
    "\n",
    "val_encodings['start_positions'] = [val_encodings.char_to_token(idx, x) if val_encodings.char_to_token(idx, x) != None else -1\n",
    "                                      for idx, x in enumerate(start_positions) ]\n",
    "val_encodings['end_positions'] = [val_encodings.char_to_token(idx, x-1) if val_encodings.char_to_token(idx, x) != None else -1\n",
    "                                    for idx, x in enumerate(end_positions)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-26T05:36:09.641950Z",
     "iopub.status.busy": "2021-06-26T05:36:09.641400Z",
     "iopub.status.idle": "2021-06-26T05:36:09.651700Z",
     "shell.execute_reply": "2021-06-26T05:36:09.650694Z",
     "shell.execute_reply.started": "2021-06-26T05:36:09.641905Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch.utils.data import Dataset, DataLoader, TensorDataset\n",
    "    \n",
    "import torch\n",
    "\n",
    "class SquadDataset(torch.utils.data.Dataset):\n",
    "    def __init__(self, encodings):\n",
    "        self.encodings = encodings\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        return {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.encodings.input_ids)\n",
    "\n",
    "train_dataset = SquadDataset(train_encodings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-26T05:36:10.298355Z",
     "iopub.status.busy": "2021-06-26T05:36:10.298080Z",
     "iopub.status.idle": "2021-06-26T05:52:46.690293Z",
     "shell.execute_reply": "2021-06-26T05:52:46.689574Z",
     "shell.execute_reply.started": "2021-06-26T05:36:10.298335Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:11: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  # This is added back by InteractiveShellApp.init_path()\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5.914609909057617 0.0 0.0\n",
      "5.220417022705078 0.125 0.0\n",
      "4.716152191162109 0.0 0.0\n",
      "3.629622459411621 0.25 0.25\n",
      "2.7960753440856934 0.375 0.25\n",
      "3.017632007598877 0.5 0.375\n",
      "2.131814479827881 0.25 0.625\n",
      "2.1228792667388916 0.5 0.25\n",
      "3.46903657913208 0.125 0.0\n",
      "2.7174510955810547 0.25 0.25\n",
      "3.371978521347046 0.25 0.25\n",
      "2.2220964431762695 0.25 0.375\n",
      "2.3144443035125732 0.5 0.5\n",
      "2.6459059715270996 0.25 0.375\n",
      "2.48060941696167 0.5 0.375\n",
      "1.8796577453613281 0.375 0.5\n",
      "2.3050951957702637 0.5 0.375\n",
      "1.8356908559799194 0.625 0.5\n",
      "1.479246973991394 0.625 0.375\n",
      "1.6082086563110352 0.5 0.75\n",
      "1.8967199325561523 0.625 0.375\n",
      "1.5556164979934692 0.375 0.5\n",
      "2.332493305206299 0.625 0.25\n",
      "1.8171426057815552 0.5 0.75\n",
      "2.13638973236084 0.5 0.375\n",
      "2.7112174034118652 0.25 0.375\n",
      "3.0005276203155518 0.375 0.25\n",
      "1.93096923828125 0.375 0.5\n",
      "1.0288045406341553 0.875 0.75\n",
      "2.2864294052124023 0.375 0.125\n",
      "1.6522210836410522 0.75 0.25\n",
      "1.6973814964294434 0.625 0.75\n",
      "2.4134974479675293 0.125 0.5\n",
      "1.4316622018814087 0.5 0.625\n",
      "2.1923255920410156 0.125 0.375\n",
      "1.3540745973587036 0.625 0.375\n",
      "1.8177225589752197 0.5 0.5\n",
      "2.033798933029175 0.375 0.375\n",
      "1.2836990356445312 0.375 0.625\n",
      "0.7430977821350098 0.625 0.625\n",
      "2.1060996055603027 0.375 0.625\n",
      "2.4395909309387207 0.5 0.625\n",
      "1.6946547031402588 0.375 0.625\n",
      "1.6582491397857666 0.625 0.375\n",
      "1.4579412937164307 0.5 0.25\n",
      "1.5582122802734375 0.5 0.5\n",
      "2.5214474201202393 0.375 0.5\n",
      "1.1383371353149414 0.5 0.75\n",
      "1.9027775526046753 0.5 0.5\n",
      "1.639087438583374 0.5 0.625\n",
      "0.8917624950408936 0.5 0.75\n",
      "1.883866548538208 0.5 0.5\n",
      "1.1945124864578247 0.625 0.5\n",
      "1.4006532430648804 0.375 0.625\n",
      "1.3866503238677979 0.5 0.875\n",
      "1.1921465396881104 0.5 0.625\n",
      "1.8594887256622314 0.375 0.25\n",
      "1.9211018085479736 0.375 0.625\n",
      "1.4101392030715942 0.5 0.5\n",
      "2.6693882942199707 0.375 0.125\n",
      "1.3605334758758545 0.5 1.0\n",
      "1.152974009513855 0.625 0.375\n",
      "1.4798551797866821 0.75 0.5\n",
      "1.7325458526611328 0.375 0.625\n",
      "1.1431829929351807 0.5 0.875\n",
      "2.004338264465332 0.125 0.375\n",
      "1.7921695709228516 0.375 0.375\n",
      "1.1184604167938232 0.75 0.375\n",
      "1.1093600988388062 0.75 0.5\n",
      "1.1951581239700317 0.625 0.625\n",
      "0.7827709317207336 0.75 0.625\n",
      "1.680673360824585 0.625 0.5\n",
      "2.016104221343994 0.5 0.5\n",
      "2.663015365600586 0.375 0.5\n",
      "1.7734265327453613 0.5 0.125\n",
      "0.8448371887207031 0.875 0.75\n",
      "1.4846479892730713 0.375 0.625\n",
      "1.237281322479248 0.625 0.5\n",
      "1.9042043685913086 0.5 0.625\n",
      "1.107244610786438 0.75 0.625\n",
      "1.3368395566940308 0.375 0.25\n",
      "0.8973419666290283 0.625 0.875\n",
      "1.5649299621582031 0.5 0.625\n",
      "1.133005142211914 0.375 0.625\n",
      "1.0948532819747925 0.625 0.75\n",
      "3.442077398300171 0.375 0.125\n",
      "1.583215355873108 0.375 0.75\n",
      "1.150711178779602 0.625 0.5\n",
      "1.449603796005249 0.375 0.5\n",
      "2.612882614135742 0.25 0.25\n",
      "1.356088638305664 0.625 0.75\n",
      "1.743706464767456 0.625 0.5\n",
      "2.9137654304504395 0.25 0.375\n",
      "2.9824106693267822 0.25 0.5\n",
      "1.3705966472625732 0.75 0.625\n",
      "2.9113619327545166 0.375 0.375\n",
      "1.4810773134231567 0.625 0.5\n",
      "1.9827723503112793 0.5 0.375\n",
      "1.7070692777633667 0.5 0.625\n",
      "1.5552632808685303 0.625 0.5\n",
      "1.2068450450897217 0.375 0.625\n",
      "2.304950714111328 0.375 0.25\n",
      "2.943427562713623 0.125 0.375\n",
      "1.9699821472167969 0.25 0.375\n",
      "0.8081175684928894 0.75 0.75\n",
      "2.5683441162109375 0.5 0.5\n",
      "1.614861011505127 0.5 0.75\n",
      "0.9523611664772034 0.75 0.875\n",
      "1.3412902355194092 0.625 0.375\n",
      "0.681111216545105 0.625 0.875\n",
      "1.6939514875411987 0.75 0.75\n",
      "1.6162524223327637 0.5 0.25\n",
      "1.5810935497283936 0.5 0.5\n",
      "1.2985135316848755 0.5 0.875\n",
      "1.6665332317352295 0.75 0.5\n",
      "1.5960986614227295 0.5 0.625\n",
      "2.3580169677734375 0.25 0.5\n",
      "2.1370396614074707 0.625 0.375\n",
      "1.2852783203125 0.375 0.625\n",
      "1.3637421131134033 0.625 0.625\n",
      "1.4369807243347168 0.5 0.625\n",
      "0.5406309366226196 0.875 1.0\n",
      "0.9856841564178467 0.75 0.75\n",
      "1.2919749021530151 0.625 0.75\n",
      "2.0837745666503906 0.75 0.5\n",
      "1.4106652736663818 0.25 0.75\n",
      "1.2976438999176025 0.625 0.5\n",
      "1.1279603242874146 0.5 0.875\n",
      "1.070791482925415 0.75 0.75\n",
      "1.1066237688064575 0.5 0.5\n",
      "1.068848729133606 0.5 0.75\n",
      "0.8697092533111572 0.75 0.375\n",
      "0.753853440284729 0.75 0.75\n",
      "0.8398517966270447 0.75 0.875\n",
      "0.6962364315986633 0.875 0.75\n",
      "1.3381339311599731 0.625 0.625\n",
      "0.9217280149459839 0.5 0.75\n",
      "0.709166407585144 0.75 0.875\n",
      "0.5655719041824341 0.875 0.75\n",
      "0.7986843585968018 0.875 0.625\n",
      "1.0043702125549316 0.75 0.625\n",
      "1.7546659708023071 0.5 0.5\n",
      "0.8376826047897339 0.875 0.75\n",
      "0.7972555160522461 0.625 0.75\n",
      "1.2311582565307617 0.5 0.625\n",
      "1.279996395111084 0.5 0.75\n",
      "1.1402723789215088 0.75 0.5\n",
      "2.1254875659942627 0.375 0.375\n",
      "0.6470641493797302 0.875 0.75\n",
      "1.2756375074386597 0.75 0.5\n",
      "1.2175140380859375 0.5 0.5\n",
      "0.8726206421852112 0.5 0.625\n",
      "1.3468632698059082 0.5 0.625\n",
      "0.925274133682251 0.875 0.625\n",
      "0.8588250279426575 0.625 0.875\n",
      "0.581047773361206 0.75 0.875\n",
      "0.842117428779602 0.375 0.75\n",
      "0.883800745010376 0.75 0.75\n",
      "0.8570733666419983 0.625 0.625\n",
      "1.9648561477661133 0.375 0.375\n",
      "1.7033145427703857 0.5 0.875\n",
      "1.3634765148162842 0.5 0.625\n",
      "0.7112200260162354 0.75 0.625\n",
      "0.9658907651901245 0.5 0.875\n",
      "1.2550463676452637 0.625 0.75\n",
      "1.868295669555664 0.625 0.25\n",
      "1.4115649461746216 0.375 0.375\n",
      "1.371068000793457 0.25 0.5\n",
      "0.9419249296188354 0.625 0.625\n",
      "0.6139960289001465 0.875 0.625\n",
      "0.8211346864700317 0.625 0.625\n",
      "0.9896498918533325 0.625 0.5\n",
      "2.220961093902588 0.375 0.375\n",
      "1.9234704971313477 0.625 0.5\n",
      "2.1733903884887695 0.375 0.625\n",
      "0.6723290681838989 0.625 0.625\n",
      "1.3549494743347168 0.375 0.375\n",
      "1.214324712753296 0.625 0.375\n",
      "1.6125078201293945 0.25 0.625\n",
      "0.9934961795806885 0.5 0.75\n",
      "0.4812208414077759 0.625 1.0\n",
      "1.2370203733444214 0.75 0.5\n",
      "1.1255908012390137 0.375 0.625\n",
      "0.6391330361366272 0.75 1.0\n",
      "0.8381053805351257 0.875 0.75\n",
      "1.4583486318588257 0.625 0.5\n",
      "1.7872490882873535 0.625 0.625\n",
      "1.6713258028030396 0.5 0.625\n",
      "0.812258243560791 0.75 0.875\n",
      "1.0042030811309814 0.5 0.5\n",
      "1.2042549848556519 0.5 0.5\n",
      "1.3645341396331787 0.25 0.625\n",
      "1.1829636096954346 0.625 0.625\n",
      "1.2516465187072754 0.375 0.625\n",
      "1.379957675933838 0.5 0.75\n",
      "1.7516019344329834 0.375 0.375\n",
      "1.2170300483703613 0.875 0.375\n",
      "1.5802900791168213 0.625 0.5\n",
      "1.6657109260559082 0.375 0.5\n",
      "1.1736445426940918 0.375 0.5\n",
      "1.07021963596344 0.375 0.75\n",
      "0.4533396065235138 1.0 1.0\n",
      "0.9079322218894958 0.5 0.625\n",
      "1.0844672918319702 0.5 0.5\n",
      "0.8506572246551514 0.875 0.875\n",
      "1.2781896591186523 0.5 0.625\n",
      "1.2219796180725098 0.5 0.625\n",
      "1.8538771867752075 0.5 0.5\n",
      "1.0763243436813354 0.625 0.5\n",
      "0.6292747259140015 0.75 0.875\n",
      "0.8616589307785034 0.625 0.875\n",
      "1.5293089151382446 0.75 0.25\n",
      "1.8025939464569092 0.5 0.375\n",
      "1.2534992694854736 0.625 0.375\n",
      "1.1185129880905151 0.5 0.75\n",
      "0.8598356246948242 0.5 0.75\n",
      "0.4728383421897888 0.875 1.0\n",
      "1.4452157020568848 0.75 0.5\n",
      "0.7623503804206848 0.75 0.875\n",
      "1.0304781198501587 0.5 0.5\n",
      "0.47354722023010254 1.0 0.875\n",
      "0.6638967394828796 0.75 0.875\n",
      "1.4630093574523926 0.5 0.625\n",
      "0.5205325484275818 0.75 0.75\n",
      "1.1568098068237305 0.5 0.625\n",
      "1.600055456161499 0.5 0.625\n",
      "0.8515949249267578 0.75 0.625\n",
      "0.7314178943634033 0.625 0.625\n",
      "1.3746380805969238 0.625 0.625\n",
      "1.2461968660354614 0.5 0.375\n",
      "1.0284464359283447 0.75 0.375\n",
      "0.9950987696647644 0.75 0.625\n",
      "1.311997652053833 0.625 0.75\n",
      "1.2360080480575562 0.625 0.375\n",
      "0.38543900847435 0.875 0.875\n",
      "1.2864964008331299 0.625 0.375\n",
      "1.8411403894424438 0.375 0.5\n",
      "1.0030056238174438 0.875 0.625\n",
      "0.8941603899002075 0.5 1.0\n",
      "1.0917353630065918 0.625 0.5\n",
      "2.2458314895629883 0.5 0.0\n",
      "1.420586347579956 0.375 0.5\n",
      "1.5667568445205688 0.5 0.375\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-53-f403c8834cf1>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     20\u001b[0m         \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput_ids\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mattention_mask\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mattention_mask\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstart_positions\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mstart_positions\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mend_positions\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mend_positions\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     21\u001b[0m         \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0moutputs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 22\u001b[0;31m         \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     23\u001b[0m         \u001b[0moptim\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     24\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.6/site-packages/torch/tensor.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(self, gradient, retain_graph, create_graph)\u001b[0m\n\u001b[1;32m    219\u001b[0m                 \u001b[0mretain_graph\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    220\u001b[0m                 create_graph=create_graph)\n\u001b[0;32m--> 221\u001b[0;31m         \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mautograd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgradient\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    222\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    223\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mregister_hook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.6/site-packages/torch/autograd/__init__.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables)\u001b[0m\n\u001b[1;32m    130\u001b[0m     Variable._execution_engine.run_backward(\n\u001b[1;32m    131\u001b[0m         \u001b[0mtensors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad_tensors_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 132\u001b[0;31m         allow_unreachable=True)  # allow_unreachable flag\n\u001b[0m\u001b[1;32m    133\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    134\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "from torch.utils.data import DataLoader\n",
    "from transformers import AdamW\n",
    "\n",
    "device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\n",
    "\n",
    "model.to(device)\n",
    "model.train()\n",
    "\n",
    "train_loader = DataLoader(train_dataset, batch_size=8, shuffle=True)\n",
    "\n",
    "optim = AdamW(model.parameters(), lr=5e-5)\n",
    "\n",
    "for epoch in range(3):\n",
    "    for idx, batch in enumerate(train_loader):\n",
    "        optim.zero_grad()\n",
    "        input_ids = batch['input_ids'].to(device)\n",
    "        attention_mask = batch['attention_mask'].to(device)\n",
    "        start_positions = batch['start_positions'].to(device)\n",
    "        end_positions = batch['end_positions'].to(device)\n",
    "        outputs = model(input_ids, attention_mask=attention_mask, \n",
    "                        start_positions=start_positions, \n",
    "                        end_positions=end_positions)\n",
    "        loss = outputs[0]\n",
    "        loss.backward()\n",
    "        optim.step()\n",
    "\n",
    "        start_pred = torch.argmax(outputs['start_logits'], dim=1)\n",
    "        end_pred = torch.argmax(outputs['end_logits'], dim=1)\n",
    "        acc1 = ( (start_pred == start_positions).sum() / len(start_pred) ).item()\n",
    "        acc2 = ( (end_pred == end_positions).sum() / len(start_pred) ).item()\n",
    "\n",
    "        if idx % 10 == 0:\n",
    "            print(loss.item(), acc1, acc2)\n",
    "            with codecs.open('log.log', 'a') as up:\n",
    "                up.write('{3}\\t{0}\\t{1}\\t{2}\\n'.format(loss.item(), acc1, acc2, \n",
    "                                                       str(epoch) + '/' + str(idx) +'/'+ str(len(train_loader))))\n",
    "\n",
    "model.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-26T05:56:23.824145Z",
     "iopub.status.busy": "2021-06-26T05:56:23.823604Z",
     "iopub.status.idle": "2021-06-26T05:56:23.835560Z",
     "shell.execute_reply": "2021-06-26T05:56:23.834750Z",
     "shell.execute_reply.started": "2021-06-26T05:56:23.824099Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def predcit(doc, query):\n",
    "    print(doc)\n",
    "    print('提问：', query)\n",
    "    item = tokenizer([doc, query], max_length=512, return_tensors='pt', truncation=True, padding=True,)\n",
    "    with torch.no_grad():\n",
    "        input_ids = torch.tensor(item['input_ids']).to(device).reshape(1, -1)\n",
    "        attention_mask = torch.tensor(item['attention_mask']).to(device).reshape(1, -1)\n",
    "        \n",
    "        outputs = model(input_ids[:, :512], attention_mask[:, :512])\n",
    "        start_pred = torch.argmax(outputs['start_logits'], dim=1)\n",
    "        end_pred = torch.argmax(outputs['end_logits'], dim=1)\n",
    "    \n",
    "    try:\n",
    "        start_pred = item.token_to_chars(0, start_pred)\n",
    "        end_pred = item.token_to_chars(0, end_pred)\n",
    "    except:\n",
    "        return ''\n",
    "    \n",
    "    if start_pred.start > end_pred.end:\n",
    "        return ''\n",
    "    else:\n",
    "        return doc[start_pred.start:end_pred.end]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-26T05:56:24.660866Z",
     "iopub.status.busy": "2021-06-26T05:56:24.660343Z",
     "iopub.status.idle": "2021-06-26T05:56:24.668920Z",
     "shell.execute_reply": "2021-06-26T05:56:24.668247Z",
     "shell.execute_reply.started": "2021-06-26T05:56:24.660821Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'paragraphs': [{'id': 'DEV_0',\n",
       "   'context': '《战国无双3》（）是由光荣和ω-force开发的战国无双系列的正统第三续作。本作以三大故事为主轴，分别是以武田信玄等人为主的《关东三国志》，织田信长等人为主的《战国三杰》，石田三成等人为主的《关原的年轻武者》，丰富游戏内的剧情。此部份专门介绍角色，欲知武器情报、奥义字或擅长攻击类型等，请至战国无双系列1.由于乡里大辅先生因故去世，不得不寻找其他声优接手。从猛将传 and Z开始。2.战国无双 编年史的原创男女主角亦有专属声优。此模式是任天堂游戏谜之村雨城改编的新增模式。本作中共有20张战场地图（不含村雨城），后来发行的猛将传再新增3张战场地图。但游戏内战役数量繁多，部分地图会有兼用的状况，战役虚实则是以光荣发行的2本「战国无双3 人物真书」内容为主，以下是相关介绍。（注：前方加☆者为猛将传新增关卡及地图。）合并本篇和猛将传的内容，村雨城模式剔除，战国史模式可直接游玩。主打两大模式「战史演武」&「争霸演武」。系列作品外传作品',\n",
       "   'qas': [{'question': '《战国无双3》是由哪两个公司合作开发的？',\n",
       "     'id': 'DEV_0_QUERY_0',\n",
       "     'answers': [{'text': '光荣和ω-force', 'answer_start': 11},\n",
       "      {'text': '光荣和ω-force', 'answer_start': 11},\n",
       "      {'text': '光荣和ω-force', 'answer_start': 11}]},\n",
       "    {'question': '男女主角亦有专属声优这一模式是由谁改编的？',\n",
       "     'id': 'DEV_0_QUERY_1',\n",
       "     'answers': [{'text': '村雨城', 'answer_start': 226},\n",
       "      {'text': '村雨城', 'answer_start': 226},\n",
       "      {'text': '任天堂游戏谜之村雨城', 'answer_start': 219}]},\n",
       "    {'question': '战国史模式主打哪两个模式？',\n",
       "     'id': 'DEV_0_QUERY_2',\n",
       "     'answers': [{'text': '「战史演武」&「争霸演武」', 'answer_start': 395},\n",
       "      {'text': '「战史演武」&「争霸演武」', 'answer_start': 395},\n",
       "      {'text': '「战史演武」&「争霸演武」', 'answer_start': 395}]}]}],\n",
       " 'id': 'DEV_0',\n",
       " 'title': '战国无双3'}"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dev['data'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-26T05:56:25.755579Z",
     "iopub.status.busy": "2021-06-26T05:56:25.755046Z",
     "iopub.status.idle": "2021-06-26T05:56:25.807612Z",
     "shell.execute_reply": "2021-06-26T05:56:25.807005Z",
     "shell.execute_reply.started": "2021-06-26T05:56:25.755534Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "《战国无双3》（）是由光荣和ω-force开发的战国无双系列的正统第三续作。本作以三大故事为主轴，分别是以武田信玄等人为主的《关东三国志》，织田信长等人为主的《战国三杰》，石田三成等人为主的《关原的年轻武者》，丰富游戏内的剧情。此部份专门介绍角色，欲知武器情报、奥义字或擅长攻击类型等，请至战国无双系列1.由于乡里大辅先生因故去世，不得不寻找其他声优接手。从猛将传 and Z开始。2.战国无双 编年史的原创男女主角亦有专属声优。此模式是任天堂游戏谜之村雨城改编的新增模式。本作中共有20张战场地图（不含村雨城），后来发行的猛将传再新增3张战场地图。但游戏内战役数量繁多，部分地图会有兼用的状况，战役虚实则是以光荣发行的2本「战国无双3 人物真书」内容为主，以下是相关介绍。（注：前方加☆者为猛将传新增关卡及地图。）合并本篇和猛将传的内容，村雨城模式剔除，战国史模式可直接游玩。主打两大模式「战史演武」&「争霸演武」。系列作品外传作品\n",
      "提问： 《战国无双3》是由哪两个公司合作开发的？\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:6: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  \n",
      "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:7: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  import sys\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'光荣和ω-force'"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predcit(dev['data'][0]['paragraphs'][0]['context'],\n",
    "       dev['data'][0]['paragraphs'][0]['qas'][0]['question'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-26T05:56:50.825781Z",
     "iopub.status.busy": "2021-06-26T05:56:50.825212Z",
     "iopub.status.idle": "2021-06-26T05:56:50.879091Z",
     "shell.execute_reply": "2021-06-26T05:56:50.878513Z",
     "shell.execute_reply.started": "2021-06-26T05:56:50.825735Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "《战国无双3》（）是由光荣和ω-force开发的战国无双系列的正统第三续作。本作以三大故事为主轴，分别是以武田信玄等人为主的《关东三国志》，织田信长等人为主的《战国三杰》，石田三成等人为主的《关原的年轻武者》，丰富游戏内的剧情。此部份专门介绍角色，欲知武器情报、奥义字或擅长攻击类型等，请至战国无双系列1.由于乡里大辅先生因故去世，不得不寻找其他声优接手。从猛将传 and Z开始。2.战国无双 编年史的原创男女主角亦有专属声优。此模式是任天堂游戏谜之村雨城改编的新增模式。本作中共有20张战场地图（不含村雨城），后来发行的猛将传再新增3张战场地图。但游戏内战役数量繁多，部分地图会有兼用的状况，战役虚实则是以光荣发行的2本「战国无双3 人物真书」内容为主，以下是相关介绍。（注：前方加☆者为猛将传新增关卡及地图。）合并本篇和猛将传的内容，村雨城模式剔除，战国史模式可直接游玩。主打两大模式「战史演武」&「争霸演武」。系列作品外传作品\n",
      "提问： 男女主角亦有专属声优这一模式是由谁改编的？\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:6: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  \n",
      "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:7: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  import sys\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'村雨城'"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predcit(dev['data'][0]['paragraphs'][0]['context'],\n",
    "       dev['data'][0]['paragraphs'][0]['qas'][1]['question'])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.6 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
